# Updated by Wei on 20140630
from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
# import matplotlib
import os
# import pylab
import random
import sys
import time
from sets import Set
# from scipy import stats
import numpy as np

# functions:
# the testing query set can be: tail5K from gov2, 200 HJ queries(09-12) from clueweb09B
# computeQPCForQueries()
# queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
# queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
# listCachingAndQueryProcessing()
# verificationCheck()
# listCaching()
# computeIndexOverallStatisticsBasedOnCurrentUnderlyingDocumentPartitionScheme()
# computeTheCPUEvaluationCostForTheCurrentMethod()

def computeQPCForQueries():
    queryTermsWithListLengthDict = {}
    # dodo:
    # inputFileName0 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/queryTermsWithTheirLengthsOfInvertedList"
    # vidaserver1:
    # clueweb09B 09_12
    # inputFileName0 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/clueweb09BQueryTermsFromHJQueries_09_12_20140624"
    # gov2 04_06
    inputFileName0 = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/gov2QueryTermsFromHJQueries_04_06_20140701"
    inputFileHandler = open(inputFileName0,"r")
    
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        currentTermID = lineElements[0]
        currentTerm = lineElements[1]
        lengthOfTheList = int(lineElements[2])
        if currentTerm not in queryTermsWithListLengthDict:
            queryTermsWithListLengthDict[currentTerm] = lengthOfTheList
    inputFileHandler.close()
    print "len(queryTermsWithListLengthDict):",len(queryTermsWithListLengthDict)
    
    queryIDWithUniqueQueryTermsDict = {}
    # dodo:
    # inputFileName1 = "/home/diaosi/filesToBeDoubt/100KQueries_tail_5k"
    # vidaserver1:
    # inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/results/dynamicUnigramFromWei/clueweb09B/09_12_topic_queries"
    inputFileName1 = "/local_scratch/wei/workspace/NYU_IRTK/results/dynamicUnigramFromWei/gov2/04-06.topics.701-850.polyIRTKCompatibleMode"
    inputFileHandler = open(inputFileName1,"r")

    # dodo:
    # outputFileName = "/home/diaosi/filesToBeDoubt/100KQueries_tail_5k_withCPUEvaluationCost_20140422"
    # vidaserver1:
    # outputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/200HJQueries_09_12_clueweb09B_withCPUEvaluationCost_20140630"
    outputFileName = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/150HJQueries_04_06_gov2_withCPUEvaluationCost_20140701"
    outputFileHandler = open(outputFileName,"w")
    
    for line in inputFileHandler.readlines():
        queryID = int(line.strip().split(":")[0])
        queryTermList = line.strip().split(":")[1].strip().split(" ")
        
        queryIDWithUniqueQueryTermsDict[queryID] = []
        
        data = ""
        for element in queryTermList:
            data += element + " "
        
        for i in range(0,len(data)):
            # print "data[i]:",ord(data[i])
            if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
                # Just replace them with a space.
                data = data[:i] + " " + data[i+1:]
        
        currentNewQueryTermList = data.strip().split(" ")
        currentNewQueryTermDict = {}
        currentQueryCPUEvaluationCost = 0
        for queryTerm in currentNewQueryTermList:
            if queryTerm.strip() != "":
                queryTermLower = queryTerm.lower()
                if queryTermLower not in currentNewQueryTermDict:
                    currentNewQueryTermDict[queryTermLower] = 1
                    queryIDWithUniqueQueryTermsDict[queryID].append(queryTermLower)
                    currentQueryCPUEvaluationCost += queryTermsWithListLengthDict[queryTermLower]
                else:
                    pass
        
        outputFileHandler.write( str(currentQueryCPUEvaluationCost) + " " + line.strip() + "\n")
    
    print "Overall:"
    print "inputFileName0:",inputFileName0
    print "inputFileName1:",inputFileName1
    print "outputFileName:",outputFileName

def queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict):
    sumQueryProcessingCost = 0
    numOfQueriesAnswered = 0
    queryFullyAnsweredDict = {}
    for i in range(95001,100000):
        currentQueryProcessingCost = 0
        queryCoveredFlag = True
        for term in queryIDWithUniqueQueryTermsDict[i]:
            if term not in termInFirstTierDict:
                queryCoveredFlag = False
            else:
                currentQueryProcessingCost += termInFirstTierDict[term]
        if queryCoveredFlag:
            numOfQueriesAnswered += 1
            queryFullyAnsweredDict[i] = 1
            sumQueryProcessingCost += currentQueryProcessingCost
            #debug 
            #print i
            #print queryIDWithUniqueQueryTermsDict[i]
            #print termInFirstTierDict["geologist"]
        else:
            sumQueryProcessingCost += 0
    
    return (numOfQueriesAnswered,sumQueryProcessingCost,queryFullyAnsweredDict)

def queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict):
    if numOfQueriesNeededToBeAnsweredInSecondTier <= 0:
        return (0,0)
    sumQPCInSecondTier = 0
    currentNumOfQueriesAnswered = 0
    inputFileName = "/home/diaosi/filesToBeDoubt/100KQueries_tail_5k_withCPUEvaluationCost_20140422_sortedByCPUEvaluationCost"
    inputFileHandler = open(inputFileName,"r")
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        currentQueryID = int(lineElements[1].strip().split(":")[0])
        if currentQueryID not in queryFullyAnsweredDict: 
            currentQueryProcessingCost = int(lineElements[0])
            sumQPCInSecondTier += currentQueryProcessingCost
            currentNumOfQueriesAnswered += 1
        else:
            # This query has been answered by the first tier
            pass
        
        if currentNumOfQueriesAnswered == numOfQueriesNeededToBeAnsweredInSecondTier:
            break
    
    inputFileHandler.close()
    return (currentNumOfQueriesAnswered,sumQPCInSecondTier)

# num of postings: 6451948010 for the whole lexicon in gov2
def listCachingAndQueryProcessing():
    # AND semantics:
    queryIDWithUniqueQueryTermsDict = {}
    inputFileName0 = "/home/diaosi/filesToBeDoubt/100KQueries_tail_5k"
    inputFileHandler = open(inputFileName0,"r")

    for line in inputFileHandler.readlines():
        queryID = int(line.strip().split(":")[0])
        queryTermList = line.strip().split(":")[1].strip().split(" ")
        
        queryIDWithUniqueQueryTermsDict[queryID] = []
        
        data = ""
        for element in queryTermList:
            data += element + " "
        
        for i in range(0,len(data)):
            # print "data[i]:",ord(data[i])
            if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
                # Just replace them with a space.
                data = data[:i] + " " + data[i+1:]
        
        currentNewQueryTermList = data.strip().split(" ")
        currentNewQueryTermDict = {}
        
        for queryTerm in currentNewQueryTermList:
            if queryTerm.strip() != "":
                queryTermLower = queryTerm.lower()
                if queryTermLower not in currentNewQueryTermDict:
                    currentNewQueryTermDict[queryTermLower] = 1
                    queryIDWithUniqueQueryTermsDict[queryID].append(queryTermLower)
                else:
                    pass
        
        # print queryID,queryIDWithUniqueQueryTermsDict[queryID]
    
    print "len(queryIDWithUniqueQueryTermsDict):",len(queryIDWithUniqueQueryTermsDict)
    inputFileHandler.close()
    
    TOTAL_NUM_OF_POSTINGS = 6451948010
    numOfPostingNeededToBePoppedAtDebugPercentage = 5000000
    numOfPostingNeededToBePoppedAt1Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.01)
    numOfPostingNeededToBePoppedAt3Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.03)
    numOfPostingNeededToBePoppedAt5Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.05)
    numOfPostingNeededToBePoppedAt10Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.1)
    numOfPostingNeededToBePoppedAt15Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.15)
    numOfPostingNeededToBePoppedAt20Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.2)
    numOfPostingNeededToBePoppedAt30Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.3)
    numOfPostingNeededToBePoppedAt40Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.4)
    numOfPostingNeededToBePoppedAt50Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.5)
    numOfPostingNeededToBePoppedAt60Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.6)
    numOfPostingNeededToBePoppedAt70Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.7)
    numOfPostingNeededToBePoppedAt80Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.8)
    numOfPostingNeededToBePoppedAt90Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.9)
    
    numOfPostingNeededToBePoppedAtDebugPercentageFlag = True
    numOfPostingNeededToBePoppedAt1PercentageFlag = True
    numOfPostingNeededToBePoppedAt3PercentageFlag = True
    numOfPostingNeededToBePoppedAt5PercentageFlag = True
    numOfPostingNeededToBePoppedAt10PercentageFlag = True
    numOfPostingNeededToBePoppedAt15PercentageFlag = True
    numOfPostingNeededToBePoppedAt20PercentageFlag = True
    numOfPostingNeededToBePoppedAt30PercentageFlag = True
    numOfPostingNeededToBePoppedAt40PercentageFlag = True
    numOfPostingNeededToBePoppedAt50PercentageFlag = True
    numOfPostingNeededToBePoppedAt60PercentageFlag = True
    numOfPostingNeededToBePoppedAt70PercentageFlag = True
    numOfPostingNeededToBePoppedAt80PercentageFlag = True
    numOfPostingNeededToBePoppedAt90PercentageFlag = True
    
    print "Debug%:",numOfPostingNeededToBePoppedAtDebugPercentage
    print "1%:",numOfPostingNeededToBePoppedAt1Percentage
    print "3%:",numOfPostingNeededToBePoppedAt3Percentage
    print "5%:",numOfPostingNeededToBePoppedAt5Percentage
    print "10%:",numOfPostingNeededToBePoppedAt10Percentage
    print "15%:",numOfPostingNeededToBePoppedAt15Percentage
    print "20%:",numOfPostingNeededToBePoppedAt20Percentage
    print "30%:",numOfPostingNeededToBePoppedAt30Percentage
    print "40%:",numOfPostingNeededToBePoppedAt40Percentage
    print "50%:",numOfPostingNeededToBePoppedAt50Percentage
    print "60%:",numOfPostingNeededToBePoppedAt60Percentage
    print "70%:",numOfPostingNeededToBePoppedAt70Percentage
    print "80%:",numOfPostingNeededToBePoppedAt80Percentage
    print "90%:",numOfPostingNeededToBePoppedAt90Percentage
    
    termInFirstTierDict = {}
    sumLengthOfList = 0
    # OLD and meaningless setting, cause NOT every query will be answered.
    # QUALITY_CONTROL_GOAL = 0.8 # maintain 80% of the document results, 0.8 overall rate
    # CURRENT setting
    QUALITY_CONTROL_GOAL = 1.0
    NUM_OF_QUERIES_PROCESSED = 4981 # num of queries actually processed under AND semantics
    NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL = int(NUM_OF_QUERIES_PROCESSED * QUALITY_CONTROL_GOAL) 
    
    inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWith_LengthOfList_FreqInHead95KQueryLog_greedySelectionValue_20140422_sortedByGreedySelectionValue"
    inputFileHandler = open(inputFileName1,"r")
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentTerm = currentLineElements[0]
        currentTermLengthOfList = int(currentLineElements[1])
        if currentTerm not in termInFirstTierDict:
            termInFirstTierDict[currentTerm] = currentTermLengthOfList
            sumLengthOfList += currentTermLengthOfList
        else:
            print "prolbem3"
            exit(1)

        if sumLengthOfList > numOfPostingNeededToBePoppedAtDebugPercentage and numOfPostingNeededToBePoppedAtDebugPercentageFlag:
            (numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,queryFullyAnsweredDict) = queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
            numOfQueriesNeededToBeAnsweredInSecondTier = NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL - numOfQueriesAnsweredInFirstTier
            (numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier) = queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
            totalNumQueriesAnswered = numOfQueriesAnsweredInFirstTier + numOfQueriesAnsweredInSecondTier
            totalCPUEvaluationCost = sumQPCInFirstTier + sumQPCInSecondTier
            print "debug%",sumLengthOfList,numOfPostingNeededToBePoppedAtDebugPercentage,len(termInFirstTierDict),numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier,totalNumQueriesAnswered,totalCPUEvaluationCost
            # print len(queryFullyAnsweredDict)
            numOfPostingNeededToBePoppedAtDebugPercentageFlag = False
            
        
        if sumLengthOfList > numOfPostingNeededToBePoppedAt1Percentage and numOfPostingNeededToBePoppedAt1PercentageFlag:
            # debug
            #print termInFirstTierDict
            #print len(termInFirstTierDict)
            #exit(1)
            (numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,queryFullyAnsweredDict) = queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
            numOfQueriesNeededToBeAnsweredInSecondTier = NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL - numOfQueriesAnsweredInFirstTier
            (numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier) = queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
            totalNumQueriesAnswered = numOfQueriesAnsweredInFirstTier + numOfQueriesAnsweredInSecondTier
            totalCPUEvaluationCost = sumQPCInFirstTier + sumQPCInSecondTier
            print "1%",sumLengthOfList,numOfPostingNeededToBePoppedAt1Percentage,len(termInFirstTierDict),numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier,totalNumQueriesAnswered,totalCPUEvaluationCost
            numOfPostingNeededToBePoppedAt1PercentageFlag = False
            

        if sumLengthOfList > numOfPostingNeededToBePoppedAt3Percentage and numOfPostingNeededToBePoppedAt3PercentageFlag:
            (numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,queryFullyAnsweredDict) = queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
            numOfQueriesNeededToBeAnsweredInSecondTier = NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL - numOfQueriesAnsweredInFirstTier
            (numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier) = queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
            totalNumQueriesAnswered = numOfQueriesAnsweredInFirstTier + numOfQueriesAnsweredInSecondTier
            totalCPUEvaluationCost = sumQPCInFirstTier + sumQPCInSecondTier
            print "3%",sumLengthOfList,numOfPostingNeededToBePoppedAt3Percentage,len(termInFirstTierDict),numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier,totalNumQueriesAnswered,totalCPUEvaluationCost
            numOfPostingNeededToBePoppedAt3PercentageFlag = False
            
        if sumLengthOfList > numOfPostingNeededToBePoppedAt5Percentage and numOfPostingNeededToBePoppedAt5PercentageFlag:
            (numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,queryFullyAnsweredDict) = queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
            numOfQueriesNeededToBeAnsweredInSecondTier = NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL - numOfQueriesAnsweredInFirstTier
            (numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier) = queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
            totalNumQueriesAnswered = numOfQueriesAnsweredInFirstTier + numOfQueriesAnsweredInSecondTier
            totalCPUEvaluationCost = sumQPCInFirstTier + sumQPCInSecondTier
            print "5%",sumLengthOfList,numOfPostingNeededToBePoppedAt5Percentage,len(termInFirstTierDict),numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier,totalNumQueriesAnswered,totalCPUEvaluationCost
            numOfPostingNeededToBePoppedAt5PercentageFlag = False
            
        if sumLengthOfList > numOfPostingNeededToBePoppedAt10Percentage and numOfPostingNeededToBePoppedAt10PercentageFlag:
            (numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,queryFullyAnsweredDict) = queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
            numOfQueriesNeededToBeAnsweredInSecondTier = NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL - numOfQueriesAnsweredInFirstTier
            (numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier) = queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
            totalNumQueriesAnswered = numOfQueriesAnsweredInFirstTier + numOfQueriesAnsweredInSecondTier
            totalCPUEvaluationCost = sumQPCInFirstTier + sumQPCInSecondTier
            print "10%",sumLengthOfList,numOfPostingNeededToBePoppedAt10Percentage,len(termInFirstTierDict),numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier,totalNumQueriesAnswered,totalCPUEvaluationCost
            numOfPostingNeededToBePoppedAt10PercentageFlag = False
            
        if sumLengthOfList > numOfPostingNeededToBePoppedAt15Percentage and numOfPostingNeededToBePoppedAt15PercentageFlag:
            (numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,queryFullyAnsweredDict) = queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
            numOfQueriesNeededToBeAnsweredInSecondTier = NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL - numOfQueriesAnsweredInFirstTier
            (numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier) = queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
            totalNumQueriesAnswered = numOfQueriesAnsweredInFirstTier + numOfQueriesAnsweredInSecondTier
            totalCPUEvaluationCost = sumQPCInFirstTier + sumQPCInSecondTier
            print "15%",sumLengthOfList,numOfPostingNeededToBePoppedAt15Percentage,len(termInFirstTierDict),numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier,totalNumQueriesAnswered,totalCPUEvaluationCost
            numOfPostingNeededToBePoppedAt15PercentageFlag = False
            
        if sumLengthOfList > numOfPostingNeededToBePoppedAt20Percentage and numOfPostingNeededToBePoppedAt20PercentageFlag:
            (numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,queryFullyAnsweredDict) = queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
            numOfQueriesNeededToBeAnsweredInSecondTier = NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL - numOfQueriesAnsweredInFirstTier
            (numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier) = queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
            totalNumQueriesAnswered = numOfQueriesAnsweredInFirstTier + numOfQueriesAnsweredInSecondTier
            totalCPUEvaluationCost = sumQPCInFirstTier + sumQPCInSecondTier
            print "20%",sumLengthOfList,numOfPostingNeededToBePoppedAt20Percentage,len(termInFirstTierDict),numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier,totalNumQueriesAnswered,totalCPUEvaluationCost
            numOfPostingNeededToBePoppedAt20PercentageFlag = False
            
        if sumLengthOfList > numOfPostingNeededToBePoppedAt30Percentage and numOfPostingNeededToBePoppedAt30PercentageFlag:
            (numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,queryFullyAnsweredDict) = queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
            numOfQueriesNeededToBeAnsweredInSecondTier = NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL - numOfQueriesAnsweredInFirstTier
            (numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier) = queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
            totalNumQueriesAnswered = numOfQueriesAnsweredInFirstTier + numOfQueriesAnsweredInSecondTier
            totalCPUEvaluationCost = sumQPCInFirstTier + sumQPCInSecondTier
            print "30%",sumLengthOfList,numOfPostingNeededToBePoppedAt30Percentage,len(termInFirstTierDict),numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier,totalNumQueriesAnswered,totalCPUEvaluationCost
            numOfPostingNeededToBePoppedAt30PercentageFlag = False
            
        if sumLengthOfList > numOfPostingNeededToBePoppedAt40Percentage and numOfPostingNeededToBePoppedAt40PercentageFlag:
            (numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,queryFullyAnsweredDict) = queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
            numOfQueriesNeededToBeAnsweredInSecondTier = NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL - numOfQueriesAnsweredInFirstTier
            (numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier) = queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
            totalNumQueriesAnswered = numOfQueriesAnsweredInFirstTier + numOfQueriesAnsweredInSecondTier
            totalCPUEvaluationCost = sumQPCInFirstTier + sumQPCInSecondTier
            print "40%",sumLengthOfList,numOfPostingNeededToBePoppedAt40Percentage,len(termInFirstTierDict),numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier,totalNumQueriesAnswered,totalCPUEvaluationCost
            numOfPostingNeededToBePoppedAt40PercentageFlag = False
        
        if sumLengthOfList > numOfPostingNeededToBePoppedAt50Percentage and numOfPostingNeededToBePoppedAt50PercentageFlag:
            (numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,queryFullyAnsweredDict) = queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
            numOfQueriesNeededToBeAnsweredInSecondTier = NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL - numOfQueriesAnsweredInFirstTier
            (numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier) = queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
            totalNumQueriesAnswered = numOfQueriesAnsweredInFirstTier + numOfQueriesAnsweredInSecondTier
            totalCPUEvaluationCost = sumQPCInFirstTier + sumQPCInSecondTier
            print "50%",sumLengthOfList,numOfPostingNeededToBePoppedAt50Percentage,len(termInFirstTierDict),numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier,totalNumQueriesAnswered,totalCPUEvaluationCost
            numOfPostingNeededToBePoppedAt50PercentageFlag = False
            
        if sumLengthOfList > numOfPostingNeededToBePoppedAt60Percentage and numOfPostingNeededToBePoppedAt60PercentageFlag:
            (numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,queryFullyAnsweredDict) = queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
            numOfQueriesNeededToBeAnsweredInSecondTier = NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL - numOfQueriesAnsweredInFirstTier
            (numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier) = queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
            totalNumQueriesAnswered = numOfQueriesAnsweredInFirstTier + numOfQueriesAnsweredInSecondTier
            totalCPUEvaluationCost = sumQPCInFirstTier + sumQPCInSecondTier
            print "60%",sumLengthOfList,numOfPostingNeededToBePoppedAt60Percentage,len(termInFirstTierDict),numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier,totalNumQueriesAnswered,totalCPUEvaluationCost
            numOfPostingNeededToBePoppedAt60PercentageFlag = False
            
        if sumLengthOfList > numOfPostingNeededToBePoppedAt70Percentage and numOfPostingNeededToBePoppedAt70PercentageFlag:
            (numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,queryFullyAnsweredDict) = queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
            numOfQueriesNeededToBeAnsweredInSecondTier = NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL - numOfQueriesAnsweredInFirstTier
            (numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier) = queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
            totalNumQueriesAnswered = numOfQueriesAnsweredInFirstTier + numOfQueriesAnsweredInSecondTier
            totalCPUEvaluationCost = sumQPCInFirstTier + sumQPCInSecondTier
            print "70%",sumLengthOfList,numOfPostingNeededToBePoppedAt70Percentage,len(termInFirstTierDict),numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier,totalNumQueriesAnswered,totalCPUEvaluationCost
            numOfPostingNeededToBePoppedAt70PercentageFlag = False
            
        if sumLengthOfList > numOfPostingNeededToBePoppedAt80Percentage and numOfPostingNeededToBePoppedAt80PercentageFlag:
            (numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,queryFullyAnsweredDict) = queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
            numOfQueriesNeededToBeAnsweredInSecondTier = NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL - numOfQueriesAnsweredInFirstTier
            (numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier) = queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
            totalNumQueriesAnswered = numOfQueriesAnsweredInFirstTier + numOfQueriesAnsweredInSecondTier
            totalCPUEvaluationCost = sumQPCInFirstTier + sumQPCInSecondTier
            print "80%",sumLengthOfList,numOfPostingNeededToBePoppedAt80Percentage,len(termInFirstTierDict),numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier,totalNumQueriesAnswered,totalCPUEvaluationCost
            numOfPostingNeededToBePoppedAt80PercentageFlag = False
            
        if sumLengthOfList > numOfPostingNeededToBePoppedAt90Percentage and numOfPostingNeededToBePoppedAt90PercentageFlag:
            (numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,queryFullyAnsweredDict) = queryProcessingInFirstTier(queryIDWithUniqueQueryTermsDict,termInFirstTierDict)
            numOfQueriesNeededToBeAnsweredInSecondTier = NUM_OF_QUERIES_NEEDED_TO_BE_ANSWERED_FULLY_IN_TOTAL - numOfQueriesAnsweredInFirstTier
            (numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier) = queryProcessingInSecondTier(numOfQueriesNeededToBeAnsweredInSecondTier,queryFullyAnsweredDict)
            totalNumQueriesAnswered = numOfQueriesAnsweredInFirstTier + numOfQueriesAnsweredInSecondTier
            totalCPUEvaluationCost = sumQPCInFirstTier + sumQPCInSecondTier
            print "90%",sumLengthOfList,numOfPostingNeededToBePoppedAt90Percentage,len(termInFirstTierDict),numOfQueriesAnsweredInFirstTier,sumQPCInFirstTier,numOfQueriesAnsweredInSecondTier,sumQPCInSecondTier,totalNumQueriesAnswered,totalCPUEvaluationCost
            numOfPostingNeededToBePoppedAt90PercentageFlag = False
            exit(1)
            
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()

def verificationCheck():
    sumOfLengthOfList = 0
    inputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWith_LengthOfList_FreqInHead95KQueryLog_greedySelectionValue_20140422_sortedByGreedySelectionValue"
    inputFileHandler = open(inputFileName,"r")
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentTermLengthOfListLength = int(currentLineElements[1])
        sumOfLengthOfList += currentTermLengthOfListLength
        currentLine = inputFileHandler.readline()
    print "sumOfLengthOfList:",sumOfLengthOfList
    inputFileHandler.close()
    
def listCaching():
    print "Begins..."
    inputFileName0 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/effectToMake1StFactor/freqOfFreqInQueries_0_1_95K_95%_with_query_terms_appended_sortedByFreqR"
    inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWithLengthOfList"
    outputFileName = "/home/diaosi/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWith_LengthOfList_FreqInHead95KQueryLog_greedySelectionValue_20140422"
    
    termFreqDict = {}
    inputFileHandler = open(inputFileName0,"r")
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        freqInQueryLog = int(currentLineElements[0])
        numOfTermsBelongingToThisClass = int(currentLineElements[1])
        
        # verification check
        if numOfTermsBelongingToThisClass != len(currentLineElements[2:]):
            if numOfTermsBelongingToThisClass != int(currentLineElements[2]):
                print "problem1"
                print "numOfTermsBelongingToThisClass:",numOfTermsBelongingToThisClass
                print "len(currentLineElements[2:]):",len(currentLineElements[2:])
                print "len(currentLineElements[2]):",len(currentLineElements[2])
                # print currentLine.strip()
                exit(1)
            else:
                pass
        
        if numOfTermsBelongingToThisClass == len(currentLineElements[2:]) and numOfTermsBelongingToThisClass != 0:
            for term in currentLineElements[2:]:
                if term not in termFreqDict:
                    termFreqDict[term] = freqInQueryLog
                else:
                    print "problem2"
                    exit(1)
        currentLine = inputFileHandler.readline() 
    inputFileHandler.close()
    print "len(termFreqDict):",len(termFreqDict)
    print "pass1"
    
    
    inputFileHandler = open(inputFileName1,"r")
    outputFileHandler = open(outputFileName,"w")
    currentLine = inputFileHandler.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentTerm = currentLineElements[0]
        currentTermLengthOfTheList = currentLineElements[1]
        currentTermFreqInQueryLog = 0
        if currentTerm not in termFreqDict:
            currentTermFreqInQueryLog = 0
        else:
            currentTermFreqInQueryLog = termFreqDict[currentTerm]
        greedySelectionValue = float(currentTermFreqInQueryLog) / float(currentTermLengthOfTheList)
        outputFileHandler.write(currentTerm + " " + str(currentTermLengthOfTheList) + " " + str(currentTermFreqInQueryLog) + " " + str(greedySelectionValue) + "\n")
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    outputFileHandler.close()
    print "Overall:"
    print "inputFileName0:",inputFileName0
    print "inputFileName1:",inputFileName1
    print "outputFileName:",outputFileName
    print "pass2"
    print "Ends."

def computeIndexOverallStatisticsBasedOnCurrentUnderlyingDocumentPartitionScheme():
    print "begins..."
    numOfDocumentsInTotalInGov2 = 25205179     
    # option2:
    # The following assignments are ONLY fitted for Xdoc document assignment scheme
    # 10 debug%
    # 16234 1%
    # 58892 3%
    # 113151 5%
    # 303078 10%
    # 573053 15%
    # 960415 20%
    # 1704697 30%
    # 3048106 40%
    # 4840867 50%
    # 7078881 60%
    # 9726244 70%
    # 13270243 80%
    # 17993779 90%
    # 25205179 100%
    numOfDocsAtDebugPercent = 10
    numOfDocsAt1Percent = 16234
    numOfDocsAt3Percent = 58892
    numOfDocsAt5Percent = 113151
    numOfDocsAt10Percent = 303078
    numOfDocsAt15Percent = 573053
    numOfDocsAt20Percent = 960415
    numOfDocsAt30Percent = 1704697
    numOfDocsAt40Percent = 3048106
    numOfDocsAt50Percent = 4840867
    numOfDocsAt60Percent = 7078881
    numOfDocsAt70Percent = 9726244
    numOfDocsAt80Percent = 13270243
    numOfDocsAt90Percent = 17993779 
    
    print "numOfDocsAtDebugPercent:",numOfDocsAtDebugPercent
    print "numOfDocsAt1Percent:",numOfDocsAt1Percent
    print "numOfDocsAt3Percent:",numOfDocsAt3Percent
    print "numOfDocsAt5Percent:",numOfDocsAt5Percent
    print "numOfDocsAt10Percent:",numOfDocsAt10Percent
    print "numOfDocsAt15Percent:",numOfDocsAt15Percent
    print "numOfDocsAt20Percent:",numOfDocsAt20Percent
    print "numOfDocsAt30Percent:",numOfDocsAt30Percent
    print "numOfDocsAt40Percent:",numOfDocsAt40Percent
    print "numOfDocsAt50Percent:",numOfDocsAt50Percent
    print "numOfDocsAt60Percent:",numOfDocsAt60Percent
    print "numOfDocsAt70Percent:",numOfDocsAt70Percent
    print "numOfDocsAt80Percent:",numOfDocsAt80Percent
    print "numOfDocsAt90Percent:",numOfDocsAt90Percent

    docIDDict0 = {}
    docIDDict0ActiveFlag = True
    numOfDocsActuallyBeenComputedInDict0 = 0
    
    docIDDict1 = {}
    docIDDict1ActiveFlag = True
    numOfDocsActuallyBeenComputedInDict1 = 0
    
    docIDDict2 = {}
    docIDDict2ActiveFlag = True
    numOfDocsActuallyBeenComputedInDict2 = 0
    
    docIDDict3 = {}
    docIDDict3ActiveFlag = True
    numOfDocsActuallyBeenComputedInDict3 = 0
    
    docIDDict4 = {}
    docIDDict4ActiveFlag = True
    numOfDocsActuallyBeenComputedInDict4 = 0
    
    docIDDict5 = {}
    docIDDict5ActiveFlag = True
    numOfDocsActuallyBeenComputedInDict5 = 0
    
    docIDDict6 = {}
    docIDDict6ActiveFlag = True
    numOfDocsActuallyBeenComputedInDict6 = 0
    
    docIDDict7 = {}
    docIDDict7ActiveFlag = True
    numOfDocsActuallyBeenComputedInDict7 = 0
    
    docIDDict8 = {}
    docIDDict8ActiveFlag = True
    numOfDocsActuallyBeenComputedInDict8 = 0
    
    docIDDict9 = {}
    docIDDict9ActiveFlag = True
    numOfDocsActuallyBeenComputedInDict9 = 0
    
    docIDDict10 = {}
    docIDDict10ActiveFlag = True
    numOfDocsActuallyBeenComputedInDict10 = 0
    
    docIDDict11 = {}
    docIDDict11ActiveFlag = True
    numOfDocsActuallyBeenComputedInDict11 = 0
    
    docIDDict12 = {}
    docIDDict12ActiveFlag = True
    numOfDocsActuallyBeenComputedInDict12 = 0
    
    docIDDict13 = {}
    docIDDict13ActiveFlag = True
    numOfDocsActuallyBeenComputedInDict13 = 0
    
    # option2: sorted by Xdoc procedure
    inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/gov2_Docs_with_TheirXdocValues_Since20140428_sortedByXdocValues"
    inputFileHanlder = open(inputFileName1,"r")
    currentLine = inputFileHanlder.readline()
    currentLineCounter = 1
    while currentLine:
        if currentLineCounter % 10000 == 0:
            print "currentLineCounter:",currentLineCounter,"processed."
            
        currentLineElements = currentLine.strip().split(" ")
        currentDocID = int(currentLineElements[1])
        
        if currentDocID not in docIDDict0 and docIDDict0ActiveFlag:
            docIDDict0[currentDocID] = 1
        

        if currentDocID not in docIDDict1 and docIDDict1ActiveFlag:
            docIDDict1[currentDocID] = 1
        

        if currentDocID not in docIDDict2 and docIDDict2ActiveFlag:
            docIDDict2[currentDocID] = 1
        

        if currentDocID not in docIDDict3 and docIDDict3ActiveFlag:
            docIDDict3[currentDocID] = 1
        

        if currentDocID not in docIDDict4 and docIDDict4ActiveFlag:
            docIDDict4[currentDocID] = 1
        

        if currentDocID not in docIDDict5 and docIDDict5ActiveFlag:
            docIDDict5[currentDocID] = 1
        

        if currentDocID not in docIDDict6 and docIDDict6ActiveFlag:
            docIDDict6[currentDocID] = 1
        

        if currentDocID not in docIDDict7 and docIDDict7ActiveFlag:
            docIDDict7[currentDocID] = 1
        

        if currentDocID not in docIDDict8 and docIDDict8ActiveFlag:
            docIDDict8[currentDocID] = 1
        

        if currentDocID not in docIDDict9 and docIDDict9ActiveFlag:
            docIDDict9[currentDocID] = 1
        
        if currentDocID not in docIDDict10 and docIDDict10ActiveFlag:
            docIDDict10[currentDocID] = 1
        
        if currentDocID not in docIDDict11 and docIDDict11ActiveFlag:
            docIDDict11[currentDocID] = 1
        
        if currentDocID not in docIDDict12 and docIDDict12ActiveFlag:
            docIDDict12[currentDocID] = 1
        
        if currentDocID not in docIDDict13 and docIDDict13ActiveFlag:
            docIDDict13[currentDocID] = 1
        
        
        # need to set the flag to False when fill up the whole dict
        if len(docIDDict0) == numOfDocsAtDebugPercent:
            docIDDict0ActiveFlag = False
         
        if len(docIDDict1) == numOfDocsAt1Percent:
            docIDDict1ActiveFlag = False
            # switch 1
            # break
        
        if len(docIDDict2) == numOfDocsAt3Percent:
            docIDDict2ActiveFlag = False
        
        if len(docIDDict3) == numOfDocsAt5Percent:
            docIDDict3ActiveFlag = False
        
        if len(docIDDict4) == numOfDocsAt10Percent:
            docIDDict4ActiveFlag = False
        
        if len(docIDDict5) == numOfDocsAt15Percent:
            docIDDict5ActiveFlag = False
        
        if len(docIDDict6) == numOfDocsAt20Percent:
            docIDDict6ActiveFlag = False
        
        if len(docIDDict7) == numOfDocsAt30Percent:
            docIDDict7ActiveFlag = False
        
        if len(docIDDict8) == numOfDocsAt40Percent:
            docIDDict8ActiveFlag = False
        
        if len(docIDDict9) == numOfDocsAt50Percent:
            docIDDict9ActiveFlag = False
        
        if len(docIDDict10) == numOfDocsAt60Percent:
            docIDDict10ActiveFlag = False
        
        if len(docIDDict11) == numOfDocsAt70Percent:
            docIDDict11ActiveFlag = False
        
        if len(docIDDict12) == numOfDocsAt80Percent:
            docIDDict12ActiveFlag = False
        
        if len(docIDDict13) == numOfDocsAt90Percent:
            docIDDict13ActiveFlag = False
     
        currentLine = inputFileHanlder.readline()
        currentLineCounter += 1
    
    print "len(docIDDict0):",len(docIDDict0)
    print "len(docIDDict1):",len(docIDDict1)
    print "len(docIDDict2):",len(docIDDict2)
    print "len(docIDDict3):",len(docIDDict3)
    print "len(docIDDict4):",len(docIDDict4)
    print "len(docIDDict5):",len(docIDDict5)
    print "len(docIDDict6):",len(docIDDict6)
    print "len(docIDDict7):",len(docIDDict7)
    print "len(docIDDict8):",len(docIDDict8)
    print "len(docIDDict9):",len(docIDDict9)
    print "len(docIDDict10):",len(docIDDict10)
    print "len(docIDDict11):",len(docIDDict11)
    print "len(docIDDict12):",len(docIDDict12)
    print "len(docIDDict13):",len(docIDDict13)
    inputFileHanlder.close()
    
    # The following dicts are corresponding to the above dict to store the actual lexicons and with their lengths
    termIDWithFreqDictForDocIDDict0 = {}
    termIDWithFreqDictForDocIDDict1 = {}
    termIDWithFreqDictForDocIDDict2 = {}
    termIDWithFreqDictForDocIDDict3 = {}
    termIDWithFreqDictForDocIDDict4 = {}
    termIDWithFreqDictForDocIDDict5 = {}
    termIDWithFreqDictForDocIDDict6 = {}
    termIDWithFreqDictForDocIDDict7 = {}
    termIDWithFreqDictForDocIDDict8 = {}
    termIDWithFreqDictForDocIDDict9 = {}
    termIDWithFreqDictForDocIDDict10 = {}
    termIDWithFreqDictForDocIDDict11 = {}
    termIDWithFreqDictForDocIDDict12 = {}
    termIDWithFreqDictForDocIDDict13 = {}
    
    # options:
    inputFileList = []
    inputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/LEAVE_selectedDocumentPostingValuesInfo_PART0_1_DOC.binary"
    inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/LEAVE_selectedDocumentPostingValuesInfo_PART2_DOC.binary"
    inputFileName4 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/LEAVE_selectedDocumentPostingValuesInfo_PART3_DOC.binary"
    inputFileName5 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/LEAVE_selectedDocumentPostingValuesInfo_PART4_DOC.binary"
    inputFileList.append(inputFileName2)
    inputFileList.append(inputFileName3)
    inputFileList.append(inputFileName4)
    inputFileList.append(inputFileName5)
    print "len(inputFileList):",len(inputFileList)
    
    for currentInputFileName in inputFileList:
        inputFileHandler = open(currentInputFileName,"rb")
        statinfo = os.stat(currentInputFileName)
        fileSize = statinfo.st_size
        
        print "currentInputFileName:",currentInputFileName
        print "file size:",fileSize
        
        numOfBytesCheck = 0
        numOfBytesRead = 0
        numOfDocsProcessed = 0
        # numOfDocsProcessed = 6000000
        
        while numOfBytesRead < fileSize:
            if numOfDocsProcessed % 100000 == 0 and numOfDocsProcessed != 0:
                print numOfDocsProcessed,numOfBytesRead
                # break
            
            # switch 2
            # if numOfDocsProcessed >= 12900:
            #    break
            
            # print "numOfBytesRead:",numOfBytesRead
            byteString = inputFileHandler.read(4 + 4)
            (docID,numOfPostings) = unpack( "2I", byteString)
            # print "docID:",str(docID),str(numOfPostings)
            
            
            docIDDict0ActiveFlag = False
            docIDDict1ActiveFlag = False
            docIDDict2ActiveFlag = False
            docIDDict3ActiveFlag = False
            docIDDict4ActiveFlag = False
            docIDDict5ActiveFlag = False
            docIDDict6ActiveFlag = False
            docIDDict7ActiveFlag = False
            docIDDict8ActiveFlag = False
            docIDDict9ActiveFlag = False
            docIDDict10ActiveFlag = False
            docIDDict11ActiveFlag = False
            docIDDict12ActiveFlag = False
            docIDDict13ActiveFlag = False
            
            if docID in docIDDict0:
                # print "process docID in dict0:",docID
                numOfDocsActuallyBeenComputedInDict0 += 1
                docIDDict0ActiveFlag = True
            if docID in docIDDict1:
                # print "process docID in dict1:",docID
                numOfDocsActuallyBeenComputedInDict1 += 1
                docIDDict1ActiveFlag = True
            if docID in docIDDict2:
                # print "process docID in dict2:",docID
                numOfDocsActuallyBeenComputedInDict2 += 1
                docIDDict2ActiveFlag = True
            if docID in docIDDict3:
                # print "process docID in dict3:",docID
                numOfDocsActuallyBeenComputedInDict3 += 1
                docIDDict3ActiveFlag = True
            if docID in docIDDict4:
                # print "process docID in dict4:",docID
                numOfDocsActuallyBeenComputedInDict4 += 1
                docIDDict4ActiveFlag = True
            if docID in docIDDict5:
                numOfDocsActuallyBeenComputedInDict5 += 1
                docIDDict5ActiveFlag = True
            if docID in docIDDict6:
                numOfDocsActuallyBeenComputedInDict6 += 1
                docIDDict6ActiveFlag = True
            if docID in docIDDict7:
                numOfDocsActuallyBeenComputedInDict7 += 1
                docIDDict7ActiveFlag = True
            if docID in docIDDict8:
                numOfDocsActuallyBeenComputedInDict8 += 1
                docIDDict8ActiveFlag = True
            if docID in docIDDict9:
                numOfDocsActuallyBeenComputedInDict9 += 1
                docIDDict9ActiveFlag = True
            if docID in docIDDict10:
                numOfDocsActuallyBeenComputedInDict10 += 1
                docIDDict10ActiveFlag = True
            if docID in docIDDict11:
                numOfDocsActuallyBeenComputedInDict11 += 1
                docIDDict11ActiveFlag = True
            if docID in docIDDict12:
                numOfDocsActuallyBeenComputedInDict12 += 1
                docIDDict12ActiveFlag = True
            if docID in docIDDict13:
                numOfDocsActuallyBeenComputedInDict13 += 1
                docIDDict13ActiveFlag = True

            # do some analysis work here
            # print "--->analyze",docID,numOfPostings
            for index in range(0,numOfPostings):
                byteString = inputFileHandler.read(4 + 4 + 4 + 4 + 4)
                (termID,staticProbability,dynamicProbability,combinedProbability,impactScore) = unpack( "1I4f", byteString)
                # staticProbability,dynamicProbability,combinedProbability,impactScore
                # print index,termID,staticProbability,dynamicProbability,combinedProbability,impactScore
                # print index,termID
                if docIDDict0ActiveFlag:
                    if termID not in termIDWithFreqDictForDocIDDict0:
                        termIDWithFreqDictForDocIDDict0[termID] = 1
                    else:
                        termIDWithFreqDictForDocIDDict0[termID] += 1
                        
                if docIDDict1ActiveFlag:
                    if termID not in termIDWithFreqDictForDocIDDict1:
                        termIDWithFreqDictForDocIDDict1[termID] = 1
                    else:
                        termIDWithFreqDictForDocIDDict1[termID] += 1
                        
                if docIDDict2ActiveFlag:
                    if termID not in termIDWithFreqDictForDocIDDict2:
                        termIDWithFreqDictForDocIDDict2[termID] = 1
                    else:
                        termIDWithFreqDictForDocIDDict2[termID] += 1
                        
                if docIDDict3ActiveFlag:
                    if termID not in termIDWithFreqDictForDocIDDict3:
                        termIDWithFreqDictForDocIDDict3[termID] = 1
                    else:
                        termIDWithFreqDictForDocIDDict3[termID] += 1
                        
                if docIDDict4ActiveFlag:
                    if termID not in termIDWithFreqDictForDocIDDict4:
                        termIDWithFreqDictForDocIDDict4[termID] = 1
                    else:
                        termIDWithFreqDictForDocIDDict4[termID] += 1
                        
                if docIDDict5ActiveFlag:
                    if termID not in termIDWithFreqDictForDocIDDict5:
                        termIDWithFreqDictForDocIDDict5[termID] = 1
                    else:
                        termIDWithFreqDictForDocIDDict5[termID] += 1
                        
                if docIDDict6ActiveFlag:
                    if termID not in termIDWithFreqDictForDocIDDict6:
                        termIDWithFreqDictForDocIDDict6[termID] = 1
                    else:
                        termIDWithFreqDictForDocIDDict6[termID] += 1
                        
                if docIDDict7ActiveFlag:
                    if termID not in termIDWithFreqDictForDocIDDict7:
                        termIDWithFreqDictForDocIDDict7[termID] = 1
                    else:
                        termIDWithFreqDictForDocIDDict7[termID] += 1
                        
                if docIDDict8ActiveFlag:
                    if termID not in termIDWithFreqDictForDocIDDict8:
                        termIDWithFreqDictForDocIDDict8[termID] = 1
                    else:
                        termIDWithFreqDictForDocIDDict8[termID] += 1
                        
                if docIDDict9ActiveFlag:
                    if termID not in termIDWithFreqDictForDocIDDict9:
                        termIDWithFreqDictForDocIDDict9[termID] = 1
                    else:
                        termIDWithFreqDictForDocIDDict9[termID] += 1
                        
                if docIDDict10ActiveFlag:
                    if termID not in termIDWithFreqDictForDocIDDict10:
                        termIDWithFreqDictForDocIDDict10[termID] = 1
                    else:
                        termIDWithFreqDictForDocIDDict10[termID] += 1                

                if docIDDict11ActiveFlag:
                    if termID not in termIDWithFreqDictForDocIDDict11:
                        termIDWithFreqDictForDocIDDict11[termID] = 1
                    else:
                        termIDWithFreqDictForDocIDDict11[termID] += 1

                if docIDDict12ActiveFlag:
                    if termID not in termIDWithFreqDictForDocIDDict12:
                        termIDWithFreqDictForDocIDDict12[termID] = 1
                    else:
                        termIDWithFreqDictForDocIDDict12[termID] += 1

                if docIDDict13ActiveFlag:
                    if termID not in termIDWithFreqDictForDocIDDict13:
                        termIDWithFreqDictForDocIDDict13[termID] = 1
                    else:
                        termIDWithFreqDictForDocIDDict13[termID] += 1
            # print termIDWithFreqDictForDocIDDict0
            # exit(1)
            
            numOfBytesRead += 4
            numOfBytesRead += 4
            numOfBytesRead += numOfPostings * 4 * 5
            
            numOfBytesCheck += 4
            numOfBytesCheck += 4
            numOfBytesCheck += numOfPostings * 4 * 5
            #debug
            #if docID == 15826653:
            #    print "find"
            #    exit(1)
            
            inputFileHandler.seek(numOfBytesRead)
            numOfDocsProcessed += 1
    
    print "numOfDocsProcessed:",numOfDocsProcessed   
    print "numOfBytesRead:",numOfBytesRead
    print "numOfBytesCheck:",numOfBytesCheck
    
    # output the terms with their length of inverted list in the sub index
    # random document partition based inverted index statistics
    # outputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_1%_20140425"
    # outputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_3%_20140425"
    # outputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_5%_20140425"
    # outputFileName4 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_10%_20140425"
    # outputFileName5 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_20%_20140425"
    # outputFileName6 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_30%_20140425"
    # outputFileName7 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_40%_20140425"
    # outputFileName8 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_50%_20140425"
    
    
    # Xdoc document partition based inverted index statistics
    # in production
    
    outputFileName0 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_debug%_20140525"
    outputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_1%_20140525"
    outputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_3%_20140525"
    outputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_5%_20140525"
    outputFileName4 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_10%_20140525"
    outputFileName5 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_15%_20140525"
    outputFileName6 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_20%_20140525"
    outputFileName7 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_30%_20140525"
    outputFileName8 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_40%_20140525"
    outputFileName9 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_50%_20140525"
    outputFileName10 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_60%_20140525"
    outputFileName11 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_70%_20140525"
    outputFileName12 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_80%_20140525"
    outputFileName13 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_90%_20140525"
    
    '''
    # in debug
    outputFileName0 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_debug%_debugMode_20140525"
    outputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_1%_debugMode_20140525"
    outputFileName2 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_3%_debugMode_20140525"
    outputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_5%_debugMode_20140525"
    outputFileName4 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_10%_debugMode_20140525"
    outputFileName5 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_15%_debugMode_20140525"
    outputFileName6 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_20%_debugMode_20140525"
    outputFileName7 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_30%_debugMode_20140525"
    outputFileName8 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_40%_debugMode_20140525"
    outputFileName9 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_50%_debugMode_20140525"
    outputFileName10 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_60%_debugMode_20140525"
    outputFileName11 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_70%_debugMode_20140525"
    outputFileName12 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_80%_debugMode_20140525"
    outputFileName13 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_90%_debugMode_20140525"
    '''
    
    outputFileHandler0 = open(outputFileName0,"w")
    outputFileHandler1 = open(outputFileName1,"w")
    outputFileHandler2 = open(outputFileName2,"w")
    outputFileHandler3 = open(outputFileName3,"w")
    outputFileHandler4 = open(outputFileName4,"w")
    outputFileHandler5 = open(outputFileName5,"w")
    outputFileHandler6 = open(outputFileName6,"w")
    outputFileHandler7 = open(outputFileName7,"w")
    outputFileHandler8 = open(outputFileName8,"w")
    outputFileHandler9 = open(outputFileName9,"w")
    outputFileHandler10 = open(outputFileName10,"w")
    outputFileHandler11 = open(outputFileName11,"w")
    outputFileHandler12 = open(outputFileName12,"w")
    outputFileHandler13 = open(outputFileName13,"w")
    
    termIDList = []
    termIDList = termIDWithFreqDictForDocIDDict0.keys()
    termIDList.sort(cmp=None, key=None, reverse=False)
    for termID in termIDList:
        outputFileHandler0.write(str(termID) + " " + str(termIDWithFreqDictForDocIDDict0[termID]) + "\n")

    termIDList = []
    termIDList = termIDWithFreqDictForDocIDDict1.keys()
    termIDList.sort(cmp=None, key=None, reverse=False)
    for termID in termIDList:
        outputFileHandler1.write(str(termID) + " " + str(termIDWithFreqDictForDocIDDict1[termID]) + "\n")

    termIDList = []
    termIDList = termIDWithFreqDictForDocIDDict2.keys()
    termIDList.sort(cmp=None, key=None, reverse=False)
    for termID in termIDList:
        outputFileHandler2.write(str(termID) + " " + str(termIDWithFreqDictForDocIDDict2[termID]) + "\n")

    termIDList = []
    termIDList = termIDWithFreqDictForDocIDDict3.keys()
    termIDList.sort(cmp=None, key=None, reverse=False)
    for termID in termIDList:
        outputFileHandler3.write(str(termID) + " " + str(termIDWithFreqDictForDocIDDict3[termID]) + "\n")
        
    termIDList = []
    termIDList = termIDWithFreqDictForDocIDDict4.keys()
    termIDList.sort(cmp=None, key=None, reverse=False)
    for termID in termIDList:
        outputFileHandler4.write(str(termID) + " " + str(termIDWithFreqDictForDocIDDict4[termID]) + "\n")
        
    termIDList = []
    termIDList = termIDWithFreqDictForDocIDDict5.keys()
    termIDList.sort(cmp=None, key=None, reverse=False)
    for termID in termIDList:
        outputFileHandler5.write(str(termID) + " " + str(termIDWithFreqDictForDocIDDict5[termID]) + "\n")

    termIDList = []
    termIDList = termIDWithFreqDictForDocIDDict6.keys()
    termIDList.sort(cmp=None, key=None, reverse=False)
    for termID in termIDList:
        outputFileHandler6.write(str(termID) + " " + str(termIDWithFreqDictForDocIDDict6[termID]) + "\n")
        
    termIDList = []
    termIDList = termIDWithFreqDictForDocIDDict7.keys()
    termIDList.sort(cmp=None, key=None, reverse=False)
    for termID in termIDList:
        outputFileHandler7.write(str(termID) + " " + str(termIDWithFreqDictForDocIDDict7[termID]) + "\n")
        
    termIDList = []
    termIDList = termIDWithFreqDictForDocIDDict8.keys()
    termIDList.sort(cmp=None, key=None, reverse=False)
    for termID in termIDList:
        outputFileHandler8.write(str(termID) + " " + str(termIDWithFreqDictForDocIDDict8[termID]) + "\n")

    termIDList = []
    termIDList = termIDWithFreqDictForDocIDDict9.keys()
    termIDList.sort(cmp=None, key=None, reverse=False)
    for termID in termIDList:
        outputFileHandler9.write(str(termID) + " " + str(termIDWithFreqDictForDocIDDict9[termID]) + "\n")

    termIDList = []
    termIDList = termIDWithFreqDictForDocIDDict10.keys()
    termIDList.sort(cmp=None, key=None, reverse=False)
    for termID in termIDList:
        outputFileHandler10.write(str(termID) + " " + str(termIDWithFreqDictForDocIDDict10[termID]) + "\n")

    termIDList = []
    termIDList = termIDWithFreqDictForDocIDDict11.keys()
    termIDList.sort(cmp=None, key=None, reverse=False)
    for termID in termIDList:
        outputFileHandler11.write(str(termID) + " " + str(termIDWithFreqDictForDocIDDict11[termID]) + "\n")

    termIDList = []
    termIDList = termIDWithFreqDictForDocIDDict12.keys()
    termIDList.sort(cmp=None, key=None, reverse=False)
    for termID in termIDList:
        outputFileHandler12.write(str(termID) + " " + str(termIDWithFreqDictForDocIDDict12[termID]) + "\n")

    termIDList = []
    termIDList = termIDWithFreqDictForDocIDDict13.keys()
    termIDList.sort(cmp=None, key=None, reverse=False)
    for termID in termIDList:
        outputFileHandler13.write(str(termID) + " " + str(termIDWithFreqDictForDocIDDict13[termID]) + "\n")
        
    
    
    print "Overall:"
    print "inputFileName1:",inputFileName1
    print "inputFileName2:",inputFileName2
    print "inputFileName3:",inputFileName3
    print "inputFileName4:",inputFileName4
    print "inputFileName5:",inputFileName5
    print "outputFileName0:",outputFileName0
    print "outputFileName1:",outputFileName1
    print "outputFileName2:",outputFileName2
    print "outputFileName3:",outputFileName3
    print "outputFileName4:",outputFileName4
    print "outputFileName5:",outputFileName5
    print "outputFileName6:",outputFileName6
    print "outputFileName7:",outputFileName7
    print "outputFileName8:",outputFileName8
    print "outputFileName9:",outputFileName9
    print "outputFileName10:",outputFileName10
    print "outputFileName11:",outputFileName11
    print "outputFileName12:",outputFileName12
    print "outputFileName13:",outputFileName13
    
    print "numOfDocsActuallyBeenComputedInDict0:",numOfDocsActuallyBeenComputedInDict0
    print "numOfDocsActuallyBeenComputedInDict1:",numOfDocsActuallyBeenComputedInDict1
    print "numOfDocsActuallyBeenComputedInDict2:",numOfDocsActuallyBeenComputedInDict2
    print "numOfDocsActuallyBeenComputedInDict3:",numOfDocsActuallyBeenComputedInDict3
    print "numOfDocsActuallyBeenComputedInDict4:",numOfDocsActuallyBeenComputedInDict4
    print "numOfDocsActuallyBeenComputedInDict5:",numOfDocsActuallyBeenComputedInDict5
    print "numOfDocsActuallyBeenComputedInDict6:",numOfDocsActuallyBeenComputedInDict6
    print "numOfDocsActuallyBeenComputedInDict7:",numOfDocsActuallyBeenComputedInDict7
    print "numOfDocsActuallyBeenComputedInDict8:",numOfDocsActuallyBeenComputedInDict8
    print "numOfDocsActuallyBeenComputedInDict9:",numOfDocsActuallyBeenComputedInDict9
    print "numOfDocsActuallyBeenComputedInDict10:",numOfDocsActuallyBeenComputedInDict10
    print "numOfDocsActuallyBeenComputedInDict11:",numOfDocsActuallyBeenComputedInDict11
    print "numOfDocsActuallyBeenComputedInDict12:",numOfDocsActuallyBeenComputedInDict12
    print "numOfDocsActuallyBeenComputedInDict13:",numOfDocsActuallyBeenComputedInDict13
    
    inputFileHandler.close()
    outputFileHandler0.close()
    outputFileHandler1.close()
    outputFileHandler2.close()
    outputFileHandler3.close()
    outputFileHandler4.close()
    outputFileHandler5.close()
    outputFileHandler6.close()
    outputFileHandler7.close()
    outputFileHandler8.close()
    outputFileHandler9.close()
    outputFileHandler10.close()
    outputFileHandler11.close()
    outputFileHandler12.close()
    outputFileHandler13.close()
    
    print "ends."
    
    '''
    # option1:
    # The following assignments are ONLY fitted for random document assignment scheme
    numOfDocsAtDebugPercent = 10
    numOfDocsAt1Percent = int(numOfDocumentsInTotalInGov2 * 0.01)
    numOfDocsAt3Percent = int(numOfDocumentsInTotalInGov2 * 0.03)
    numOfDocsAt5Percent = int(numOfDocumentsInTotalInGov2 * 0.05)
    numOfDocsAt10Percent = int(numOfDocumentsInTotalInGov2 * 0.1)
    numOfDocsAt15Percent = int(numOfDocumentsInTotalInGov2 * 0.15)
    numOfDocsAt20Percent = int(numOfDocumentsInTotalInGov2 * 0.2)
    numOfDocsAt30Percent = int(numOfDocumentsInTotalInGov2 * 0.3)
    numOfDocsAt40Percent = int(numOfDocumentsInTotalInGov2 * 0.4)
    numOfDocsAt50Percent = int(numOfDocumentsInTotalInGov2 * 0.5)
    numOfDocsAt60Percent = int(numOfDocumentsInTotalInGov2 * 0.6)
    numOfDocsAt70Percent = int(numOfDocumentsInTotalInGov2 * 0.7)
    numOfDocsAt80Percent = int(numOfDocumentsInTotalInGov2 * 0.8)
    numOfDocsAt90Percent = int(numOfDocumentsInTotalInGov2 * 0.9)
    '''
    
    '''
    # option1: random document method procedure
    inputFileName1 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/randomDocumentsSelectedAtDifferentPercentage_20140412"
    inputFileHanlder = open(inputFileName1,"r")
    currentLine = inputFileHanlder.readline()
    while currentLine:
        currentLineElements = currentLine.strip().split(" ")
        currentDocIDIndex = currentLineElements[0]
        currentDocID = int(currentLineElements[1])
        
        if currentDocID not in docIDDict:
            docIDDict[currentDocID] = 1
        else:
            print "duplicated docID detected."
            exit(1)
        
        if len(docIDDict) == numOfDocsAt50Percent:
            break
        currentLine = inputFileHanlder.readline()
    docIDList = docIDDict.keys()
    docIDList.sort(cmp=None, key=None, reverse=False)
    print "len(docIDDict):",len(docIDDict)
    print "len(docIDList):",len(docIDList)
    inputFileHanlder.close()
    '''
    
    '''
    # optional load
    termIDWithTermDict = {}
    inputFileName0 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWithTermID"
    inputFileHandler = open(inputFileName0,"r")
    currentLine = inputFileHandler.readline()
    lineNumber = 1
    while currentLine:
        if lineNumber % 1000000 == 0:
            print lineNumber,"processed."
        currentLineElements = currentLine.strip().split(" ")
        termID = currentLineElements[0]
        term = currentLineElements[1]
        if termID not in termIDWithTermDict:
            termIDWithTermDict[termID] = term
        currentLine = inputFileHandler.readline()
        lineNumber += 1
    inputFileHandler.close()
    '''

def computeTheCPUEvaluationCostForTheCurrentMethod():
    queryTermFreqDict = {}
    # 150 human judge queries
    # inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/realFreqOfTermsFor150HumanJudgedQueries"
    # tail 5k queries
    inputFileName1 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/realFreqOfTermsForTail5KFrom100K"
    inputFileHandler = open(inputFileName1,"r")
    for line in inputFileHandler.readlines():
        lineElements = line.strip().split(" ")
        queryTerm = lineElements[0]
        queryTermFreq = int(lineElements[1])
        if queryTerm not in queryTermFreqDict:
            queryTermFreqDict[queryTerm] = queryTermFreq  
    inputFileHandler.close()
    print "len(queryTermFreqDict):",len(queryTermFreqDict)
    
    queryTermIDFreqDict = {}
    inputFileName2 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWithTermID"
    inputFileHandler = open(inputFileName2,"r")
    currentLine = inputFileHandler.readline() 
    while currentLine:
        currentLineElements = currentLine.strip().split(" ") 
        currentQueryTermID = currentLineElements[0]
        currentQueryTerm = currentLineElements[1]
        if currentQueryTerm in queryTermFreqDict:
             queryTermIDFreqDict[currentQueryTermID] = queryTermFreqDict[currentQueryTerm]
        currentLine = inputFileHandler.readline()
    inputFileHandler.close()
    print "len(queryTermIDFreqDict):",len(queryTermIDFreqDict)
    # print "queryTermFreqDict['000']:",queryTermFreqDict['000']
    # print "queryTermIDFreqDict['2']:",queryTermIDFreqDict['2']
    
    inputFileList = []
    # previously computed, months ago on 2014/05/26 afternoon at school
    # inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_1%_20140425"
    # inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_3%_20140425"
    # inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_5%_20140425"
    # inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_10%_20140425"
    # inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_15%_20140425"
    # inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_20%_20140425"
    # inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_30%_20140425"
    # inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_40%_20140425"
    # inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/randomDocumentPartitionBasedInvertedIndexStatistics_50%_20140425"
    
    # the WHOLE big original thing, when computing the CPU evaluation cost using the second tier
    # inputFileName3 = "/home/diaosi/gov2ClearYourMindAndDoItAgain/wholeLexiconTermsWithLengthOfList"
    
    # the xDoc related thing, computed on 2014/05/25 the whole day, in order to see the CPU evaluation cost when using this method
    inputFileName3 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_1%_20140525"
    inputFileName4 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_3%_20140525"
    inputFileName5 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_5%_20140525"
    inputFileName6 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_10%_20140525"
    inputFileName7 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_15%_20140525"
    inputFileName8 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_20%_20140525"
    inputFileName9 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_30%_20140525"
    inputFileName10 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_40%_20140525"
    inputFileName11 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_50%_20140525"
    inputFileName12 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_60%_20140525"
    inputFileName13 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_70%_20140525"
    inputFileName14 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_80%_20140525"
    inputFileName15 = "/home/diaosi/workspace/web-search-engine-wei-2014-March/data/xDocDocumentPartitionBasedInvertedIndexStatistics_90%_20140525"
    inputFileList.append(inputFileName3)
    inputFileList.append(inputFileName4)
    inputFileList.append(inputFileName5)
    inputFileList.append(inputFileName6)
    inputFileList.append(inputFileName7)
    inputFileList.append(inputFileName8)
    inputFileList.append(inputFileName9)
    inputFileList.append(inputFileName10)
    inputFileList.append(inputFileName11)
    inputFileList.append(inputFileName12)
    inputFileList.append(inputFileName13)
    inputFileList.append(inputFileName14)
    inputFileList.append(inputFileName15)
    
    for currentInputFileName in inputFileList: 
        sumOfCPUCost = 0
        inputFileHandler = open(currentInputFileName,"r")
        currentLine = inputFileHandler.readline()
        currentLineNumber = 0
        while currentLine:
            currentLineElements = currentLine.strip().split(" ")
            # option0:
            # currentTermID = str(currentLineNumber)
            # option1:
            currentTermID = currentLineElements[0]
            currentTermIDListLength = int(currentLineElements[1])
            if currentTermID not in queryTermIDFreqDict:
                pass
            else:
                sumOfCPUCost += queryTermIDFreqDict[currentTermID] * currentTermIDListLength
             
            currentLine = inputFileHandler.readline()
            currentLineNumber += 1
        
        inputFileHandler.close()
        print "Overall:"
        print "sumOfCPUCost:",sumOfCPUCost
        print "inputFileName1:",inputFileName1
        print "inputFileName2:",inputFileName2
        print "currentInputFileName:",currentInputFileName

print "Program Begins..."
# verificationCheck()
# listCachingAndQueryProcessing()
computeQPCForQueries()
# The following functions are useful in tiering project
# Updated by Wei on 2014/05/25 afternoon
# computeIndexOverallStatisticsBasedOnCurrentUnderlyingDocumentPartitionScheme()
# Updated by Wei on 2014/05/26 afternoon
# just the cost of query processing
# computeTheCPUEvaluationCostForTheCurrentMethod()
print "Program Ends."











